{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hidden Trigger Backdoor Attack Example\n",
    "This notebook shows how to use the ART implementation of the Hidden Trigger Backdoor attack to poison a model. The goal of the attack is to create poisoned inputs within a target class with no visible trigger that can be used during model finetuning to poison the model. After finetuning, adding a trigger to certain inputs not in the target class will cause the model to reliably misclassify those inputs as the target class. Full details of the attack can be found at https://arxiv.org/abs/1910.00033."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Temporary directory not created yet\n",
      "Temporary directory: /tmp/tmphtn3n705\n"
     ]
    }
   ],
   "source": [
    "import os, sys\n",
    "from os.path import abspath\n",
    "\n",
    "module_path = os.path.abspath(os.path.join('..'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import tempfile\n",
    "# Create a temporary directory for the model checkpoint. Remove the exising one if this cell is rerun\n",
    "try:\n",
    "    temp_model_dir.cleanup()\n",
    "except NameError:\n",
    "    print(\"Temporary directory not created yet\")\n",
    "finally:\n",
    "    temp_model_dir = tempfile.TemporaryDirectory() \n",
    "    print(\"Temporary directory:\", temp_model_dir.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset and Model\n",
    "We will load the CIFAR10 dataset and a pre-trained alexnet model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from art.utils import load_dataset\n",
    "(x_train, y_train), (x_test, y_test), min_, max_ = load_dataset('cifar10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 15, 15, 64)        18496     \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 13, 13, 64)        36928     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 512)               1180160   \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 1,250,858\n",
      "Trainable params: 1,250,858\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-02-07 20:10:50.473308: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:10:50.485830: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:10:50.486699: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "tf.compat.v1.disable_eager_execution()\n",
    "from tensorflow.keras import layers, models, losses\n",
    "\n",
    "model = models.Sequential()\n",
    "\n",
    "# Create Keras convolutional neural network - basic architecture from Keras examples\n",
    "# Source here: https://github.com/keras-team/keras/blob/master/examples/cifar10_cnn.py\n",
    "model.add(layers.Conv2D(32, (3, 3), padding=\"same\", activation='relu', input_shape=x_train.shape[1:]))\n",
    "model.add(layers.Conv2D(32, (3, 3), activation='relu'))\n",
    "model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "\n",
    "model.add(layers.Conv2D(64, (3, 3), padding=\"same\", activation='relu'))\n",
    "model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
    "model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(512, activation='relu'))\n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(10))\n",
    "\n",
    "model.compile(loss=losses.CategoricalCrossentropy(from_logits=True), optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "    \n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from art.estimators.classification import KerasClassifier\n",
    "\n",
    "classifier = KerasClassifier(clip_values=(min_, max_), model=model, use_logits=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-02-07 20:10:52.502701: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-02-07 20:10:52.504157: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:10:52.505121: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:10:52.505934: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:10:54.801342: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:10:54.802255: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:10:54.803061: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:10:54.803837: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15405 MB memory:  -> device: 0, name: NVIDIA Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-02-07 20:10:57.292304: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8201\n",
      "2022-02-07 20:10:59.859147: I tensorflow/core/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000/50000 [==============================] - 10s 206us/sample - loss: 1.5992 - accuracy: 0.4125\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 4s 80us/sample - loss: 1.2345 - accuracy: 0.5570\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 4s 80us/sample - loss: 1.0482 - accuracy: 0.6261\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 4s 80us/sample - loss: 0.9228 - accuracy: 0.6748\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 4s 80us/sample - loss: 0.8412 - accuracy: 0.7046\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 4s 81us/sample - loss: 0.7769 - accuracy: 0.7259\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 4s 80us/sample - loss: 0.7238 - accuracy: 0.7431\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 4s 80us/sample - loss: 0.6842 - accuracy: 0.7603\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 4s 80us/sample - loss: 0.6479 - accuracy: 0.7711\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 4s 81us/sample - loss: 0.6216 - accuracy: 0.7815\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-02-07 20:11:42.015730: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:11:42.016131: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:11:42.016372: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:11:42.016680: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:11:42.016944: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-07 20:11:42.017193: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15405 MB memory:  -> device: 0, name: NVIDIA Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0\n",
      "2022-02-07 20:11:42.577615: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /tmp/tmphtn3n705/keras_model/assets\n"
     ]
    }
   ],
   "source": [
    "classifier.fit(x_train, y_train, nb_epochs=10, batch_size=128)\n",
    "model.save(temp_model_dir.name + '/htbd_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on benign test examples: 77.47%\n"
     ]
    }
   ],
   "source": [
    "predictions = classifier.predict(x_test)\n",
    "accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)\n",
    "print(\"Accuracy on benign test examples: {}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate the Poison\n",
    "We now will generate the poison using the hidden trigger backdoor attack. First, we define the target and source classes as well as the backdoor trigger. The target class will be the class we want to insert poisoned data into. The source class will be the class we will add a trigger to in order to cause misclassification into the target.\n",
    "\n",
    "The backdoor trigger will be a small image patch inserted into the source class images. At test time, we should be able to use this trigger to cause the classifier to misclassify source class images with the trigger added as the target class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from art.attacks.poisoning.backdoor_attack import PoisoningAttackBackdoor\n",
    "target = np.array([0,0,0,0,1,0,0,0,0,0])\n",
    "source = np.array([0,0,0,1,0,0,0,0,0,0])\n",
    "\n",
    "# Backdoor Trigger Parameters\n",
    "patch_size = 8\n",
    "x_shift = 32 - patch_size - 5\n",
    "y_shift = 32 - patch_size - 5\n",
    "\n",
    "# Define the backdoor poisoning object. Calling backdoor.poison(x) will insert the trigger into x.\n",
    "from art.attacks.poisoning import perturbations\n",
    "def mod(x):\n",
    "    original_dtype = x.dtype\n",
    "    x = perturbations.insert_image(x, backdoor_path=\"../../utils/data/backdoors/htbd.png\",\n",
    "                                   channels_first=False, random=False, x_shift=x_shift, y_shift=y_shift,\n",
    "                                   size=(patch_size,patch_size), mode='RGB', blend=1)\n",
    "    return x.astype(original_dtype)\n",
    "backdoor = PoisoningAttackBackdoor(mod)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we run the attack. `eps` controls how much the target images can be perturbed with respect to an l-infinity distance. `feature_layer` dicates with layer's output will be used to define the attack's loss. It can either be the name of the layer or the layer index according to the ART estimator. `poison_percent` controls how many poisoned samples will be generated based on the size of the input data.\n",
    "\n",
    "The attack will return poisoned inputs of the target class and the indicies in the data that those poisoned inputs should replace."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48bcdc5399ba4a2e8bfb42d9203d29ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Hidden Trigger:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | batch: 0 | i:     0 | LR: 0.01000 | Loss Val: 3060.881 | Loss Avg: 3060.881\n",
      "Epoch:  0 | batch: 0 | i:   100 | LR: 0.01000 | Loss Val: 316.473 | Loss Avg: 447.504\n",
      "Epoch:  0 | batch: 0 | i:   200 | LR: 0.01000 | Loss Val: 302.197 | Loss Avg: 377.031\n",
      "Epoch:  0 | batch: 0 | i:   300 | LR: 0.01000 | Loss Val: 282.379 | Loss Avg: 349.939\n",
      "Epoch:  0 | batch: 0 | i:   400 | LR: 0.01000 | Loss Val: 289.825 | Loss Avg: 333.681\n",
      "Epoch:  0 | batch: 0 | i:   500 | LR: 0.01000 | Loss Val: 280.297 | Loss Avg: 323.425\n",
      "Epoch:  0 | batch: 0 | i:   600 | LR: 0.01000 | Loss Val: 276.790 | Loss Avg: 316.031\n",
      "Epoch:  0 | batch: 0 | i:   700 | LR: 0.01000 | Loss Val: 277.243 | Loss Avg: 310.722\n",
      "Epoch:  0 | batch: 0 | i:   800 | LR: 0.01000 | Loss Val: 278.212 | Loss Avg: 306.634\n",
      "Epoch:  0 | batch: 0 | i:   900 | LR: 0.01000 | Loss Val: 281.881 | Loss Avg: 303.701\n",
      "Epoch:  0 | batch: 0 | i:  1000 | LR: 0.00100 | Loss Val: 278.842 | Loss Avg: 301.080\n",
      "Epoch:  0 | batch: 0 | i:  1100 | LR: 0.00100 | Loss Val: 274.151 | Loss Avg: 298.656\n",
      "Epoch:  0 | batch: 0 | i:  1200 | LR: 0.00100 | Loss Val: 274.040 | Loss Avg: 296.610\n",
      "Epoch:  0 | batch: 0 | i:  1300 | LR: 0.00100 | Loss Val: 273.977 | Loss Avg: 294.873\n",
      "Epoch:  0 | batch: 0 | i:  1400 | LR: 0.00100 | Loss Val: 273.930 | Loss Avg: 293.380\n",
      "Epoch:  0 | batch: 0 | i:  1500 | LR: 0.00100 | Loss Val: 273.895 | Loss Avg: 292.083\n",
      "Epoch:  0 | batch: 0 | i:  1600 | LR: 0.00100 | Loss Val: 273.863 | Loss Avg: 290.946\n",
      "Epoch:  0 | batch: 0 | i:  1700 | LR: 0.00100 | Loss Val: 273.838 | Loss Avg: 289.941\n",
      "Epoch:  0 | batch: 0 | i:  1800 | LR: 0.00100 | Loss Val: 273.815 | Loss Avg: 289.046\n",
      "Epoch:  0 | batch: 0 | i:  1900 | LR: 0.00100 | Loss Val: 273.795 | Loss Avg: 288.244\n",
      "Epoch:  0 | batch: 0 | i:  2000 | LR: 0.00010 | Loss Val: 273.777 | Loss Avg: 287.522\n",
      "Epoch:  0 | batch: 0 | i:  2100 | LR: 0.00010 | Loss Val: 273.764 | Loss Avg: 286.867\n",
      "Epoch:  0 | batch: 0 | i:  2200 | LR: 0.00010 | Loss Val: 273.762 | Loss Avg: 286.272\n",
      "Epoch:  0 | batch: 0 | i:  2300 | LR: 0.00010 | Loss Val: 273.760 | Loss Avg: 285.728\n",
      "Epoch:  0 | batch: 0 | i:  2400 | LR: 0.00010 | Loss Val: 273.758 | Loss Avg: 285.229\n",
      "Epoch:  0 | batch: 0 | i:  2500 | LR: 0.00010 | Loss Val: 273.757 | Loss Avg: 284.771\n",
      "Epoch:  0 | batch: 0 | i:  2600 | LR: 0.00010 | Loss Val: 273.755 | Loss Avg: 284.347\n",
      "Epoch:  0 | batch: 0 | i:  2700 | LR: 0.00010 | Loss Val: 273.753 | Loss Avg: 283.955\n",
      "Epoch:  0 | batch: 0 | i:  2800 | LR: 0.00010 | Loss Val: 273.752 | Loss Avg: 283.591\n",
      "Epoch:  0 | batch: 0 | i:  2900 | LR: 0.00010 | Loss Val: 273.750 | Loss Avg: 283.252\n",
      "Max_Loss: 273.748956231997\n",
      "Epoch:  0 | batch: 1 | i:     0 | LR: 0.01000 | Loss Val: 2555.124 | Loss Avg: 283.695\n",
      "Epoch:  0 | batch: 1 | i:   100 | LR: 0.01000 | Loss Val: 274.176 | Loss Avg: 285.714\n",
      "Epoch:  0 | batch: 1 | i:   200 | LR: 0.01000 | Loss Val: 242.080 | Loss Avg: 284.742\n",
      "Epoch:  0 | batch: 1 | i:   300 | LR: 0.01000 | Loss Val: 242.079 | Loss Avg: 283.368\n",
      "Epoch:  0 | batch: 1 | i:   400 | LR: 0.01000 | Loss Val: 238.056 | Loss Avg: 282.028\n",
      "Epoch:  0 | batch: 1 | i:   500 | LR: 0.01000 | Loss Val: 233.415 | Loss Avg: 280.719\n",
      "Epoch:  0 | batch: 1 | i:   600 | LR: 0.01000 | Loss Val: 234.263 | Loss Avg: 279.443\n",
      "Epoch:  0 | batch: 1 | i:   700 | LR: 0.01000 | Loss Val: 232.453 | Loss Avg: 278.189\n",
      "Epoch:  0 | batch: 1 | i:   800 | LR: 0.01000 | Loss Val: 232.767 | Loss Avg: 276.997\n",
      "Epoch:  0 | batch: 1 | i:   900 | LR: 0.01000 | Loss Val: 231.780 | Loss Avg: 275.858\n",
      "Epoch:  0 | batch: 1 | i:  1000 | LR: 0.00100 | Loss Val: 233.730 | Loss Avg: 274.786\n",
      "Epoch:  0 | batch: 1 | i:  1100 | LR: 0.00100 | Loss Val: 230.297 | Loss Avg: 273.705\n",
      "Epoch:  0 | batch: 1 | i:  1200 | LR: 0.00100 | Loss Val: 230.169 | Loss Avg: 272.670\n",
      "Epoch:  0 | batch: 1 | i:  1300 | LR: 0.00100 | Loss Val: 230.120 | Loss Avg: 271.683\n",
      "Epoch:  0 | batch: 1 | i:  1400 | LR: 0.00100 | Loss Val: 230.160 | Loss Avg: 270.740\n",
      "Epoch:  0 | batch: 1 | i:  1500 | LR: 0.00100 | Loss Val: 230.139 | Loss Avg: 269.839\n",
      "Epoch:  0 | batch: 1 | i:  1600 | LR: 0.00100 | Loss Val: 230.189 | Loss Avg: 268.977\n",
      "Epoch:  0 | batch: 1 | i:  1700 | LR: 0.00100 | Loss Val: 230.133 | Loss Avg: 268.151\n",
      "Epoch:  0 | batch: 1 | i:  1800 | LR: 0.00100 | Loss Val: 230.070 | Loss Avg: 267.359\n",
      "Epoch:  0 | batch: 1 | i:  1900 | LR: 0.00100 | Loss Val: 230.073 | Loss Avg: 266.600\n",
      "Epoch:  0 | batch: 1 | i:  2000 | LR: 0.00010 | Loss Val: 230.067 | Loss Avg: 265.871\n",
      "Epoch:  0 | batch: 1 | i:  2100 | LR: 0.00010 | Loss Val: 229.977 | Loss Avg: 265.167\n",
      "Epoch:  0 | batch: 1 | i:  2200 | LR: 0.00010 | Loss Val: 229.976 | Loss Avg: 264.490\n",
      "Epoch:  0 | batch: 1 | i:  2300 | LR: 0.00010 | Loss Val: 229.974 | Loss Avg: 263.839\n",
      "Epoch:  0 | batch: 1 | i:  2400 | LR: 0.00010 | Loss Val: 229.974 | Loss Avg: 263.212\n",
      "Epoch:  0 | batch: 1 | i:  2500 | LR: 0.00010 | Loss Val: 229.970 | Loss Avg: 262.608\n",
      "Epoch:  0 | batch: 1 | i:  2600 | LR: 0.00010 | Loss Val: 229.964 | Loss Avg: 262.026\n",
      "Epoch:  0 | batch: 1 | i:  2700 | LR: 0.00010 | Loss Val: 229.970 | Loss Avg: 261.463\n",
      "Epoch:  0 | batch: 1 | i:  2800 | LR: 0.00010 | Loss Val: 229.977 | Loss Avg: 260.921\n",
      "Epoch:  0 | batch: 1 | i:  2900 | LR: 0.00010 | Loss Val: 229.977 | Loss Avg: 260.396\n",
      "Max_Loss: 229.97768664230625\n",
      "Epoch:  0 | batch: 2 | i:     0 | LR: 0.01000 | Loss Val: 3025.898 | Loss Avg: 260.355\n",
      "Epoch:  0 | batch: 2 | i:   100 | LR: 0.01000 | Loss Val: 337.451 | Loss Avg: 263.796\n",
      "Epoch:  0 | batch: 2 | i:   200 | LR: 0.01000 | Loss Val: 294.471 | Loss Avg: 264.628\n",
      "Epoch:  0 | batch: 2 | i:   300 | LR: 0.01000 | Loss Val: 288.235 | Loss Avg: 265.133\n",
      "Epoch:  0 | batch: 2 | i:   400 | LR: 0.01000 | Loss Val: 279.633 | Loss Avg: 265.411\n",
      "Epoch:  0 | batch: 2 | i:   500 | LR: 0.01000 | Loss Val: 300.027 | Loss Avg: 265.659\n",
      "Epoch:  0 | batch: 2 | i:   600 | LR: 0.01000 | Loss Val: 288.801 | Loss Avg: 265.743\n",
      "Epoch:  0 | batch: 2 | i:   700 | LR: 0.01000 | Loss Val: 270.230 | Loss Avg: 265.839\n",
      "Epoch:  0 | batch: 2 | i:   800 | LR: 0.01000 | Loss Val: 266.513 | Loss Avg: 265.981\n",
      "Epoch:  0 | batch: 2 | i:   900 | LR: 0.01000 | Loss Val: 289.055 | Loss Avg: 266.105\n",
      "Epoch:  0 | batch: 2 | i:  1000 | LR: 0.00100 | Loss Val: 300.855 | Loss Avg: 266.254\n",
      "Epoch:  0 | batch: 2 | i:  1100 | LR: 0.00100 | Loss Val: 244.290 | Loss Avg: 265.982\n",
      "Epoch:  0 | batch: 2 | i:  1200 | LR: 0.00100 | Loss Val: 242.996 | Loss Avg: 265.670\n",
      "Epoch:  0 | batch: 2 | i:  1300 | LR: 0.00100 | Loss Val: 242.325 | Loss Avg: 265.354\n",
      "Epoch:  0 | batch: 2 | i:  1400 | LR: 0.00100 | Loss Val: 241.763 | Loss Avg: 265.038\n",
      "Epoch:  0 | batch: 2 | i:  1500 | LR: 0.00100 | Loss Val: 241.283 | Loss Avg: 264.725\n",
      "Epoch:  0 | batch: 2 | i:  1600 | LR: 0.00100 | Loss Val: 241.100 | Loss Avg: 264.416\n",
      "Epoch:  0 | batch: 2 | i:  1700 | LR: 0.00100 | Loss Val: 240.845 | Loss Avg: 264.111\n",
      "Epoch:  0 | batch: 2 | i:  1800 | LR: 0.00100 | Loss Val: 240.534 | Loss Avg: 263.811\n",
      "Epoch:  0 | batch: 2 | i:  1900 | LR: 0.00100 | Loss Val: 240.478 | Loss Avg: 263.516\n",
      "Epoch:  0 | batch: 2 | i:  2000 | LR: 0.00010 | Loss Val: 239.725 | Loss Avg: 263.222\n",
      "Epoch:  0 | batch: 2 | i:  2100 | LR: 0.00010 | Loss Val: 239.593 | Loss Avg: 262.931\n",
      "Epoch:  0 | batch: 2 | i:  2200 | LR: 0.00010 | Loss Val: 239.561 | Loss Avg: 262.646\n",
      "Epoch:  0 | batch: 2 | i:  2300 | LR: 0.00010 | Loss Val: 239.546 | Loss Avg: 262.368\n",
      "Epoch:  0 | batch: 2 | i:  2400 | LR: 0.00010 | Loss Val: 239.515 | Loss Avg: 262.096\n",
      "Epoch:  0 | batch: 2 | i:  2500 | LR: 0.00010 | Loss Val: 239.487 | Loss Avg: 261.830\n",
      "Epoch:  0 | batch: 2 | i:  2600 | LR: 0.00010 | Loss Val: 239.478 | Loss Avg: 261.570\n",
      "Epoch:  0 | batch: 2 | i:  2700 | LR: 0.00010 | Loss Val: 239.450 | Loss Avg: 261.316\n",
      "Epoch:  0 | batch: 2 | i:  2800 | LR: 0.00010 | Loss Val: 239.426 | Loss Avg: 261.068\n",
      "Epoch:  0 | batch: 2 | i:  2900 | LR: 0.00010 | Loss Val: 239.420 | Loss Avg: 260.824\n",
      "Max_Loss: 239.397277651261\n",
      "Number of poison samples generated: 75\n"
     ]
    }
   ],
   "source": [
    "from art.attacks.poisoning import HiddenTriggerBackdoor\n",
    "poison_attack = HiddenTriggerBackdoor(classifier, eps=16/255, target=target, source=source, feature_layer=9, backdoor=backdoor, learning_rate=0.01, decay_coeff = .1, decay_iter = 1000, max_iter=3000, batch_size=25, poison_percent=.015)\n",
    "\n",
    "poison_data, poison_indices = poison_attack.poison(x_train, y_train)\n",
    "print(\"Number of poison samples generated:\", len(poison_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finetune the Model\n",
    "Now, we must finetune the model using the poisoned data and a small number of clean training inputs.  Here, we randomly select an equal number of training inputs from each of the classes. If the poison doesn't succeed, just run the notebook again."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create finetuning dataset\n",
    "dataset_size = 2500\n",
    "num_classes = 10\n",
    "num_per_class = dataset_size/num_classes\n",
    "\n",
    "poison_dataset_inds = []\n",
    "\n",
    "for i in range(num_classes):\n",
    "    class_inds = np.where(np.argmax(y_train,axis=1) == i)[0]\n",
    "    num_select = int(num_per_class)\n",
    "    if np.argmax(target) == i:\n",
    "        num_select = int(num_select - min(num_per_class,len(poison_data)))\n",
    "        poison_dataset_inds.append(poison_indices)\n",
    "        \n",
    "    if num_select != 0:\n",
    "        poison_dataset_inds.append(np.random.choice(class_inds, num_select, replace=False))\n",
    "    \n",
    "poison_dataset_inds = np.concatenate(poison_dataset_inds)\n",
    "\n",
    "poison_x = np.copy(x_train)\n",
    "poison_x[poison_indices] = poison_data\n",
    "poison_x = poison_x[poison_dataset_inds]\n",
    "\n",
    "poison_y = np.copy(y_train)[poison_dataset_inds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 15, 15, 64)        18496     \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 13, 13, 64)        36928     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 512)               1180160   \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 1,250,858\n",
      "Trainable params: 0\n",
      "Non-trainable params: 1,250,858\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = tf.keras.models.load_model(temp_model_dir.name + '/htbd_model')\n",
    "temp_model_dir.cleanup() # Remove the temporary directory after loading the checkpoint\n",
    "\n",
    "model.trainable = False\n",
    "model.compile(loss=losses.CategoricalCrossentropy(from_logits=True), optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on benign test examples: 77.47%\n"
     ]
    }
   ],
   "source": [
    "classifier = KerasClassifier(clip_values=(min_, max_), model=model)\n",
    "predictions = classifier.predict(x_test)\n",
    "accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)\n",
    "print(\"Accuracy on benign test examples: {}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_input (InputLayer)    [(None, 32, 32, 3)]       0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 15, 15, 64)        18496     \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 13, 13, 64)        36928     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 512)               1180160   \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 1,250,858\n",
      "Trainable params: 5,130\n",
      "Non-trainable params: 1,245,728\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "finetune_model = tf.keras.layers.Dense(10)(model.layers[-2].output)\n",
    "finetune_model = tf.keras.Model(inputs=model.inputs, outputs=finetune_model)\n",
    "finetune_model.summary()\n",
    "\n",
    "lr = 0.5\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=lr)\n",
    "\n",
    "finetune_model.compile(loss=losses.CategoricalCrossentropy(from_logits=True), optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "finetune_classifier = KerasClassifier(clip_values=(min_, max_), model=finetune_model, use_logits=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on benign test examples: 6.069999999999999%\n",
      "Accuracy on benign trigger test examples: 1.5%\n",
      "Accuracy on poison trigger test examples: 1.5%\n",
      "Success on poison trigger test examples: 2.3%\n",
      "\n",
      "Training Epoch 0\n",
      "Train on 2500 samples\n",
      "2500/2500 [==============================] - 0s 103us/sample - loss: 6.7402 - accuracy: 0.6548\n",
      "Accuracy on benign test examples: 72.47%\n",
      "Accuracy on benign trigger test examples: 45.800000000000004%\n",
      "Accuracy on poison trigger test examples: 13.200000000000001%\n",
      "Success on poison trigger test examples: 50.3%\n",
      "\n",
      "Training Epoch 1\n",
      "Train on 2500 samples\n",
      "2500/2500 [==============================] - 0s 85us/sample - loss: 6.8900 - accuracy: 0.7152\n",
      "Accuracy on benign test examples: 74.87%\n",
      "Accuracy on benign trigger test examples: 60.099999999999994%\n",
      "Accuracy on poison trigger test examples: 18.099999999999998%\n",
      "Success on poison trigger test examples: 54.0%\n",
      "\n",
      "Training Epoch 2\n",
      "Train on 2500 samples\n",
      "2500/2500 [==============================] - 0s 89us/sample - loss: 5.2740 - accuracy: 0.7464\n",
      "Accuracy on benign test examples: 75.14999999999999%\n",
      "Accuracy on benign trigger test examples: 57.3%\n",
      "Accuracy on poison trigger test examples: 16.8%\n",
      "Success on poison trigger test examples: 53.7%\n",
      "\n",
      "Training Epoch 3\n",
      "Train on 2500 samples\n",
      "2500/2500 [==============================] - 0s 94us/sample - loss: 5.4636 - accuracy: 0.7536\n"
     ]
    }
   ],
   "source": [
    "trigger_test_inds = np.where(np.all(y_test == source, axis=1))[0]\n",
    "\n",
    "test_poisoned_samples, test_poisoned_labels  = backdoor.poison(x_test[trigger_test_inds], y_test[trigger_test_inds])\n",
    "\n",
    "\n",
    "for i in range(4):\n",
    "    predictions = finetune_classifier.predict(x_test)\n",
    "    accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)\n",
    "    print(\"Accuracy on benign test examples: {}%\".format(accuracy * 100))\n",
    "    \n",
    "    predictions = finetune_classifier.predict(x_test[trigger_test_inds])\n",
    "    b_accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test[trigger_test_inds], axis=1)) / len(trigger_test_inds)\n",
    "    print(\"Accuracy on benign trigger test examples: {}%\".format(b_accuracy * 100))\n",
    "    \n",
    "    predictions = finetune_classifier.predict(test_poisoned_samples)\n",
    "    p_accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(test_poisoned_labels,axis=1)) / len(test_poisoned_labels)\n",
    "    print(\"Accuracy on poison trigger test examples: {}%\".format(p_accuracy * 100))\n",
    "    p_success = np.sum(np.argmax(predictions, axis=1) == np.argmax(target)) / len(test_poisoned_labels)\n",
    "    print(\"Success on poison trigger test examples: {}%\".format(p_success * 100))\n",
    "    print()\n",
    "    print(\"Training Epoch\", i)\n",
    "    if i != 0:\n",
    "        lr *= 0.1\n",
    "        optimizer = tf.keras.optimizers.Adam(learning_rate=lr)\n",
    "        finetune_model.compile(loss=losses.CategoricalCrossentropy(from_logits=True), optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "    finetune_classifier = KerasClassifier(clip_values=(min_, max_), model=finetune_model, use_logits=True)\n",
    "\n",
    "    finetune_classifier.fit(poison_x, poison_y, nb_epochs=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Performance\n",
      "Accuracy on benign test examples: 75.16000000000001%\n",
      "Accuracy on benign trigger test examples: 57.099999999999994%\n",
      "Accuracy on poison trigger test examples: 16.8%\n",
      "Success on poison trigger test examples: 53.7%\n"
     ]
    }
   ],
   "source": [
    "print(\"Final Performance\")\n",
    "predictions = finetune_classifier.predict(x_test)\n",
    "accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)\n",
    "print(\"Accuracy on benign test examples: {}%\".format(accuracy * 100))\n",
    "\n",
    "predictions = finetune_classifier.predict(x_test[trigger_test_inds])\n",
    "b_accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test[trigger_test_inds], axis=1)) / len(trigger_test_inds)\n",
    "print(\"Accuracy on benign trigger test examples: {}%\".format(b_accuracy * 100))\n",
    "\n",
    "predictions = finetune_classifier.predict(test_poisoned_samples)\n",
    "p_accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(test_poisoned_labels,axis=1)) / len(test_poisoned_labels)\n",
    "print(\"Accuracy on poison trigger test examples: {}%\".format(p_accuracy * 100))\n",
    "p_success = np.sum(np.argmax(predictions, axis=1) == np.argmax(target)) / len(test_poisoned_labels)\n",
    "print(\"Success on poison trigger test examples: {}%\".format(p_success * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
